{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib qt5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Attach a lot of summaries to a Tensor (for TensorBoard visualization).\n",
    "def variable_summaries(var):\n",
    "    with tf.name_scope('summaries'):\n",
    "        mean = tf.reduce_mean(var)\n",
    "        tf.summary.scalar('mean', mean)\n",
    "        with tf.name_scope('stddev'):\n",
    "            stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))\n",
    "        tf.summary.scalar('stddev', stddev)\n",
    "        tf.summary.scalar('max', tf.reduce_max(var))\n",
    "        tf.summary.scalar('min', tf.reduce_min(var))\n",
    "        tf.summary.histogram('histogram', var)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nn_layer(input_tensor, input_dim, output_dim, layer_name, act, name='nn_layer'):\n",
    "    # Adding a name scope ensures logical grouping of the layers in the graph.\n",
    "    with tf.name_scope(layer_name):\n",
    "        with tf.name_scope('weights'):\n",
    "            weights = tf.Variable(tf.random_normal([input_dim, output_dim]),dtype=tf.float32)\n",
    "            variable_summaries(weights)\n",
    "        with tf.name_scope('biases'):\n",
    "            biases = tf.Variable(tf.zeros([1,output_dim]) + 0.1)\n",
    "            variable_summaries(biases)\n",
    "        with tf.name_scope('Wx_plus_b'):\n",
    "            preactivate = tf.matmul(input_tensor, weights) + biases\n",
    "            tf.summary.histogram('pre_activations', preactivate)\n",
    "            #Wx_plus_b = tf.matmul(inputs, weights) + biases\n",
    "        if act is None:\n",
    "            activations = preactivate\n",
    "        else:\n",
    "            activations = act(preactivate, name='activation')\n",
    "            tf.summary.histogram('activations', activations)\n",
    "            activations = act(preactivate)\n",
    "        return activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_data = np.linspace(-1,1,300,dtype='float32')[:,np.newaxis]\n",
    "noise = np.random.normal(0,0.05,x_data.shape)\n",
    "y_data = np.square(x_data) - 0.5 + noise\n",
    "with tf.name_scope('inputs'):\n",
    "    xs = tf.placeholder(tf.float32,[None,1],name='x_input')\n",
    "    ys = tf.placeholder(tf.float32,[None,1],name='labels')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure = plt.figure()\n",
    "ax = figure.add_subplot(1,1,1)\n",
    "ax.scatter(x_data,y_data)\n",
    "plt.ion()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer1 = nn_layer(x_data, 1, 10, 'layer1', act=tf.nn.relu, name=\"layer1\")\n",
    "predict_layer = nn_layer(layer1, 10, 1, 'layer2', act=None, name=\"predict_layer\")\n",
    "\n",
    "with tf.name_scope('loss'):\n",
    "    loss = tf.reduce_mean(tf.square(y_data - predict_layer))\n",
    "    tf.summary.scalar('loss', loss)\n",
    "with tf.name_scope('train'):\n",
    "    train_step = tf.train.AdamOptimizer(0.001).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feed_dict():\n",
    "    return {xs : x_data, ys : y_data}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    tf.global_variables_initializer().run()\n",
    "    # Merge all the summaries and write them out to directory\n",
    "    merged = tf.summary.merge_all()\n",
    "    train_writer = tf.summary.FileWriter('logs/2_layer', sess.graph)\n",
    "    for iter in range(2000):\n",
    "        sess.run(train_step)\n",
    "        if(iter % 10 == 0):\n",
    "            #print(iter,sess.run(loss))\n",
    "            try:\n",
    "                ax.lines.remove(lines[0])\n",
    "            except Exception:\n",
    "                pass\n",
    "            prediction_value = sess.run(predict_layer)\n",
    "            lines = ax.plot(x_data,prediction_value,'r-',lw=5)\n",
    "            plt.pause(0.1)\n",
    "        if(iter % 50 == 0):\n",
    "            summary = sess.run(merged, feed_dict = feed_dict())\n",
    "            train_writer.add_summary(summary, iter)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
